{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23861"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "pred_data_1=[]\n",
    "pred_num_1=0\n",
    "with open(\"./predict_res kg.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        pred_data_1.append(json.loads(d))\n",
    "        pred_num_1+=len(pred_data_1[-1]['spo_list'])\n",
    "pred_num_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [
     1,
     11,
     34
    ]
   },
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "def _is_whitespace(char):\n",
    "    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n",
    "    # \\t, \\n, and \\r are technically contorl characters but we treat them\n",
    "    # as whitespace since they are generally considered as such.\n",
    "    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat == \"Zs\":\n",
    "        return True\n",
    "    return False\n",
    "def is_chinese_char(cp):\n",
    "    \"\"\"Checks whether CP is the codepoint of a CJK character.\"\"\"\n",
    "    # This defines a \"chinese character\" as anything in the CJK Unicode block:\n",
    "    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)\n",
    "    #\n",
    "    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,\n",
    "    # despite its name. The modern Korean Hangul alphabet is a different block,\n",
    "    # as is Japanese Hiragana and Katakana. Those alphabets are used to write\n",
    "    # space-separated words, so they are not treated specially and handled\n",
    "    # like the all of the other languages.\n",
    "    if (\n",
    "        (cp >= 0x4E00 and cp <= 0x9FFF)\n",
    "        or (cp >= 0x3400 and cp <= 0x4DBF)  #\n",
    "        or (cp >= 0x20000 and cp <= 0x2A6DF)  #\n",
    "        or (cp >= 0x2A700 and cp <= 0x2B73F)  #\n",
    "        or (cp >= 0x2B740 and cp <= 0x2B81F)  #\n",
    "        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #\n",
    "        or (cp >= 0xF900 and cp <= 0xFAFF)\n",
    "        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #\n",
    "    ):  #\n",
    "        return True\n",
    "\n",
    "    return False\n",
    "def is_punctuation(char):\n",
    "    \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n",
    "    cp = ord(char)\n",
    "    # We treat all non-letter/number ASCII as punctuation.\n",
    "    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n",
    "    # Punctuation class but we treat them as punctuation anyways, for\n",
    "    # consistency.\n",
    "    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat.startswith(\"P\"):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for idx,e in enumerate(pred_data_1):\n",
    "    for spo in e['spo_list']:\n",
    "        while spo['subject'][0]=='《' and ( spo['subject'][-1]=='》' or  spo['subject'].find(\"》\")==-1):\n",
    "            print(\"sub\",spo['subject'])\n",
    "            spo['subject']=spo['subject'].replace(\"《\",\"\").replace(\"》\",\"\")\n",
    "            print(\"sub\",spo['subject'])         \n",
    "            break\n",
    "        poped_keys=[]\n",
    "        for key,value in spo['object'].items():\n",
    "            if len(value)==0:\n",
    "                print(spo)\n",
    "                poped_keys.append(key)\n",
    "            if len(poped_keys)>0:\n",
    "                for t in poped_keys:\n",
    "                    print(t)\n",
    "                    spo['object'].pop(t)\n",
    "                    spo['object_type'].pop(t)\n",
    "                print(idx,pred_data_1[idx])\n",
    "                break\n",
    "            while value[0]=='《' and (value[-1]=='》' or value.find(\"》\")==-1):\n",
    "                print(\"ob\",key,value)\n",
    "                spo['object'][key]=value.replace(\"《\",\"\").replace(\"》\",\"\")\n",
    "                print(\"ob\",spo['object'][key])        \n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./predict_res kg.json\",\"w\") as w:\n",
    "    for e in pred_data_1:\n",
    "        w.write(json.dumps(e, ensure_ascii=False)+'\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17964"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_data_2=[]\n",
    "pred_num_2=0\n",
    "with open(\"./frzhu/new_test1_res kg.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        pred_data_2.append(json.loads(d))\n",
    "        pred_num_2+=len(pred_data_2[-1]['spo_list'])\n",
    "pred_num_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_contain(a,b):\n",
    "    if a.find(b)!=-1 or b.find(a)!=-1:\n",
    "        return True\n",
    "    return False\n",
    "for idx in range(len(pred_data_1)):\n",
    "    for spo in pred_data_2[idx]['spo_list']:\n",
    "        flag=True\n",
    "        for temp in pred_data_1[idx]['spo_list']:\n",
    "            if temp['subject']==spo['subject'] and temp['predicate']==spo['predicate']  and cross_contain(temp['object']['@value'],spo['object']['@value'])\\\n",
    "               and len(temp['object']['@value'])>len(spo['object']['@value']) and len(spo['object']['@value'])<4:\n",
    "                flag=False\n",
    "                temp['object']['@value']=spo['object']['@value']\n",
    "                break\n",
    "#         if flag:\n",
    "#             pred_data_1[idx]['spo_list'].append(spo)\n",
    "# 7\n",
    "# 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./predict_res kg combine.json\",\"w\") as w:\n",
    "    for e in pred_data_1:\n",
    "        w.write(json.dumps(e, ensure_ascii=False)+'\\n')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fjw",
   "language": "python",
   "name": "fjw"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
